﻿using System;
using System.Collections;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Windows.Forms;
using System.Drawing;
using MySql.Data.MySqlClient;

namespace iLikeLite
{
    class Crawler
    {
        private string website = "http://oldnavy.gap.com/";

        public string loadedHtml;
        public Boolean stop = false;

        public AutoResetEvent broswerEvent = new AutoResetEvent(false);

        public event EventHandler CrawlerMessage;
        public event EventHandler BrowserLoad;

        public void Start()
        {
            Thread t = new Thread(new ThreadStart(StartThreaded));
            t.Start();
        }

        private void SendMessage(string msg)
        {
            if (CrawlerMessage != null)
            {
                CrawlerMessage(msg, EventArgs.Empty);
            }
        }

        private void StartThreaded()
        {
            MySqlConnection conn = new MySqlConnection(Properties.Settings.Default.mydbConnectionString);
            conn.Open();
            //ArrayList temp;

            SendMessage("Starting the crawler...");

            // get the main page HTML
            string mainPageHtml = GetSubstring(GetHtml(website), "<div class=\"topNavBottomRow\">", "</div>");

            // get the division IDs
            ArrayList divisionIds = GetIds(mainPageHtml);

            //hack - just skip to the men's division
            divisionIds.RemoveAt(1);
            divisionIds.RemoveAt(1);

            // *********TEMP*********
            //temp = new ArrayList();
            //temp.Add(divisionIds[0]);
            //divisionIds = temp;
            // *********TEMP*********

            SendMessage("Iterating through divisions...");
            // iterate through division IDs
            foreach (string divId in divisionIds)
            {
                string categoryHtml = GetSubstring(GetHtml("http://oldnavy.gap.com/browse/division.do?cid=" + divId), "Shop by Category", "<li class=\"header\"");
                ArrayList categoryIds = GetIds(categoryHtml);

                // *********TEMP*********
                //temp = new ArrayList();
                //temp.Add(categoryIds[0]);
                //categoryIds = temp;
                // *********TEMP*********

                SendMessage("Iterating through categories...");
                foreach (string catId in categoryIds)
                {
                    string productsHtml = GetHtml("http://oldnavy.gap.com/browse/category.do?cid=" + catId);
                    ArrayList productIds = GetProductIds(productsHtml);

                    // *********TEMP*********
                    //temp = new ArrayList();
                    //temp.Add(productIds[0]);
                    //productIds = temp;
                    // *********TEMP*********

                    SendMessage("Iterating through products...");
                    foreach (string prodId in productIds)
                    {
                        if (stop)
                        {
                            return;
                        }
                        string productHtml = GetHtml("http://oldnavy.gap.com/browse/product.do?" + prodId);
                        ArrayList images = GetImages(productHtml);
                        if (images.Count > 0)
                        {
                            string imageUrl = this.website + (string)images[0];
                            SendMessage("Downloading " + imageUrl);

                            string title = GetTitle(productHtml);
                            string lTitle = title.ToLower();
                            MySqlCommand cmd = new MySqlCommand("SELECT id FROM images WHERE title = ?t", conn);
                            cmd.Parameters.AddWithValue("?t", title);
                            MySqlDataReader reader = cmd.ExecuteReader();

                            if (!reader.Read() && !(lTitle.Contains("bikini") || lTitle.Contains("sunglasses") || lTitle.Contains(" bra") || lTitle.Contains("swim") || lTitle.Contains("bottom")))
                            {
                                reader.Close();
                                Image prodImage = DownloadImage(imageUrl);
                                ArrayList colors = GetColors(productHtml);

                                cmd = new MySqlCommand("INSERT INTO images (image, title, keywords) VALUES (?image, ?title, ?keywords)", conn);
                                cmd.Parameters.AddWithValue("?image", Convert.ToBase64String(ImageToByte2(prodImage)));
                                cmd.Parameters.AddWithValue("?title", title);
                                cmd.Parameters.AddWithValue("?keywords", (string)colors[0]);

                                if (cmd.ExecuteNonQuery() == 1)
                                {
                                    SendMessage("Successfully entered image into database.");
                                }
                                else
                                {
                                    SendMessage("Unable to enter image into database.");
                                }
                            }
                            else
                            {
                                reader.Close();
                            }
                        }
                        else
                        {
                            SendMessage("No image found on page.");
                        }
                    }
                }
            }
            conn.Close();
        }

        private string GetHtml(string url)
        {
            SendMessage("Loading " + url);
            BrowserLoad(url, EventArgs.Empty);
            broswerEvent.WaitOne();
            SendMessage("Done!");

            return this.loadedHtml;
        }

        private string GetSubstring(string input, string begin, string end)
        {
            input = input.Substring(input.IndexOf(begin));
            input = input.Substring(0, input.IndexOf(end));
            return input;
        }

        private ArrayList GetIds(string html)
        {
            ArrayList divisionIds = new ArrayList();
            Regex regex = new Regex("/browse/[a-z]+.do\\?cid=(?<cid>[0-9]+)");
            MatchCollection matchCol = regex.Matches(html);
            foreach (Match m in matchCol)
            {
                divisionIds.Add(m.Groups["cid"].Value);
            }
            return divisionIds;
        }

        private ArrayList GetProductIds(string html)
        {
            ArrayList productIds = new ArrayList();
            Regex regex = new Regex("/browse/product.do\\?cid=(?<cid>[0-9]+)&amp;vid=[0-9]+&amp;pid=(?<pid>[0-9]+)");
            MatchCollection matchCol = regex.Matches(html);
            foreach (Match m in matchCol)
            {
                string productId = "cid=" + m.Groups["cid"].Value + "&pid=" + m.Groups["pid"].Value;
                if (!productIds.Contains(productId))
                {
                    productIds.Add(productId);
                }
            }
            return productIds;
        }

        private ArrayList GetImages(string html)
        {
            ArrayList images = new ArrayList();
            Regex regex = new Regex("'(?<img>[/\\-_a-zA-Z0-9]+\\.jpg)'");
            MatchCollection matchCol = regex.Matches(html);
            foreach (Match m in matchCol)
            {
                images.Add(m.Groups["img"].Value);
            }
            return images;
        }

        private ArrayList GetColors(string html)
        {
            ArrayList colors = new ArrayList();
            Regex regex = new Regex("(?<color>[ a-zA-Z]+)- \\$[0-9]+\\.[0-9]+ <br />");
            MatchCollection matchCol = regex.Matches(html);
            foreach (Match m in matchCol)
            {
                colors.Add(m.Groups["color"].Value.Trim());
            }
            return colors;
        }

        private string GetTitle(string html)
        {
            string title = "";
            Regex regex = new Regex("<title>(?<title>[^|]+)");
            MatchCollection matchCol = regex.Matches(html);
            if (matchCol.Count > 0)
            {
                title = WebUtility.HtmlDecode(matchCol[0].Groups["title"].Value.Trim());
            }
            return title;
        }

        /// <summary>
        /// Function to download Image from website
        /// </summary>
        /// <param name="_URL">URL address to download image</param>
        /// <returns>Image</returns>
        public Image DownloadImage(string _URL)
        {
            Image _tmpImage = null;

            try
            {
                // Open a connection
                System.Net.HttpWebRequest _HttpWebRequest = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(_URL);

                _HttpWebRequest.AllowWriteStreamBuffering = true;

                // You can also specify additional header values like the user agent or the referer: (Optional)
                _HttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";
                _HttpWebRequest.Referer = "http://www.google.com/";

                // set timeout for 20 seconds (Optional)
                _HttpWebRequest.Timeout = 20000;

                // Request response:
                System.Net.WebResponse _WebResponse = _HttpWebRequest.GetResponse();

                // Open data stream:
                System.IO.Stream _WebStream = _WebResponse.GetResponseStream();

                // convert webstream to image
                _tmpImage = Image.FromStream(_WebStream);

                // Cleanup
                _WebResponse.Close();
                _WebResponse.Close();
            }
            catch (Exception _Exception)
            {
                // Error
                Console.WriteLine("Exception caught in process: {0}", _Exception.ToString());
                return null;
            }

            return _tmpImage;
        }

        public static byte[] ImageToByte2(Image img)
        {
            byte[] byteArray = new byte[0];
            using (MemoryStream stream = new MemoryStream())
            {
                img.Save(stream, System.Drawing.Imaging.ImageFormat.Jpeg);
                stream.Close();

                byteArray = stream.ToArray();
            }
            return byteArray;
        }
    }
}
